Behavioral Clonning

This notebook is for Udacity's Behavioral Clonning Project

Load the Data

Here I load the data provided by Udacity

(as an alternative there is also another set of data)

!git clone https://github.com/KansaiUser/BehavioralCloneData
#!git clone https://github.com/KansaiUser/BehavioralCloningTrackData

Cloning into 'BehavioralCloneData'...
remote: Enumerating objects: 24112, done.
remote: Total 24112 (delta 0), reused 0 (delta 0), pack-reused 24112
Receiving objects: 100% (24112/24112), 305.31 MiB | 38.72 MiB/s, done.
Checking out files: 100% (24109/24109), done.

!ls BehavioralCloneData/
#!ls BehavioralCloningTrackData/

driving_log.csv  IMG

The data is loaded in the current directory of the Google Cloud Machine

Import the necessary libraries

import os
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import keras
from keras.models import Sequential
from keras.optimizers import Adam
from keras.layers import Convolution2D, Conv2D, MaxPooling2D, Dropout, Flatten, Dense
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from imgaug import augmenters as iaa
import cv2
import pandas as pd
import ntpath
import random

Get the Data into a Dataframe ready to be used, and show some statistics

folder= "BehavioralCloneData"
#folder= "BehavioralCloningTrackData"
data=pd.read_csv(os.path.join(folder,"driving_log.csv"))
data.head()

	center	left	right	steering	throttle	brake	speed
0	IMG/center_2016_12_01_13_30_48_287.jpg	IMG/left_2016_12_01_13_30_48_287.jpg	IMG/right_2016_12_01_13_30_48_287.jpg	0.0	0.0	0.0	22.148290
1	IMG/center_2016_12_01_13_30_48_404.jpg	IMG/left_2016_12_01_13_30_48_404.jpg	IMG/right_2016_12_01_13_30_48_404.jpg	0.0	0.0	0.0	21.879630
2	IMG/center_2016_12_01_13_31_12_937.jpg	IMG/left_2016_12_01_13_31_12_937.jpg	IMG/right_2016_12_01_13_31_12_937.jpg	0.0	0.0	0.0	1.453011
3	IMG/center_2016_12_01_13_31_13_037.jpg	IMG/left_2016_12_01_13_31_13_037.jpg	IMG/right_2016_12_01_13_31_13_037.jpg	0.0	0.0	0.0	1.438419
4	IMG/center_2016_12_01_13_31_13_177.jpg	IMG/left_2016_12_01_13_31_13_177.jpg	IMG/right_2016_12_01_13_31_13_177.jpg	0.0	0.0	0.0	1.418236

num_bins = 25
samples_per_bin = 800
hist, bins = np.histogram(data['steering'], num_bins)

len(data)

8036

plt.hist(data['steering'],bins)
plt.plot((np.min(data['steering']), np.max(data['steering'])), (samples_per_bin, samples_per_bin))

[<matplotlib.lines.Line2D at 0x7f162e59a828>]

As you can see the data is pretty much accumulated in the center, so I limit the number of data per bin

remove_list = []
for j in range(num_bins):
  list_ = []
  for i in range(len(data['steering'])):
    if data['steering'][i] >= bins[j] and data['steering'][i] <= bins[j+1]:
      list_.append(i)
  list_ = shuffle(list_)
  list_ = list_[samples_per_bin:]
  remove_list.extend(list_)

print(len(remove_list))

4080

print('removed:', len(remove_list))
data.drop(data.index[remove_list], inplace=True)
print('remaining:', len(data))

removed: 4080
remaining: 3956

plt.hist(data['steering'],bins)

(array([  4.,   1.,   2.,   2.,   4.,  18.,  55.,  55., 174., 193., 532.,
        656., 800., 527., 577., 122., 169.,  26.,  25.,   6.,   5.,   1.,
          0.,   0.,   2.]),
 array([-0.9426954 , -0.86498758, -0.78727977, -0.70957195, -0.63186414,
        -0.55415632, -0.4764485 , -0.39874069, -0.32103287, -0.24332506,
        -0.16561724, -0.08790942, -0.01020161,  0.06750621,  0.14521402,
         0.22292184,  0.30062966,  0.37833747,  0.45604529,  0.5337531 ,
         0.61146092,  0.68916874,  0.76687655,  0.84458437,  0.92229218,
         1.        ]),
 <a list of 25 Patch objects>)

Seen this data we can see that the data is still pretty much inclined toward the center. I suspect that this will make driving in close turns a bit difficult, and I will need to supplement this later by augmenting the data

Getting Training and Validation Data

First we get arrays of the images and the steering data

data.iloc[0]

center      IMG/center_2016_12_01_13_31_13_037.jpg
left          IMG/left_2016_12_01_13_31_13_037.jpg
right        IMG/right_2016_12_01_13_31_13_037.jpg
steering                                         0
throttle                                         0
brake                                            0
speed                                      1.43842
Name: 3, dtype: object

def get_image_steering(folder,df):
  image_path_list=[]
  steering_list=[]
  for i in range(len(df)):  #not data
    image_path=df.iloc[i].center.strip()
    image_path_list.append(os.path.join(folder,image_path))
    steering= df.iloc[i].steering
    steering_list.append(steering)

    image_path=df.iloc[i].left.strip()
    image_path_list.append(os.path.join(folder,image_path))
    steering= df.iloc[i].steering+0.25 #0.15  #Try 0.25
    steering_list.append(steering)

    image_path=df.iloc[i].right.strip()
    image_path_list.append(os.path.join(folder,image_path))
    steering= df.iloc[i].steering-0.25  #0.15  #Try -0.25
    steering_list.append(steering)


  image_path_array=np.array(image_path_list)
  steering_array=np.array(steering_list)
  return image_path_array,steering_array

im,st=get_image_steering(folder,data)
print(im)
print(st)

['BehavioralCloneData/IMG/center_2016_12_01_13_31_13_037.jpg'
 'BehavioralCloneData/IMG/left_2016_12_01_13_31_13_037.jpg'
 'BehavioralCloneData/IMG/right_2016_12_01_13_31_13_037.jpg' ...
 'BehavioralCloneData/IMG/center_2016_12_01_13_46_38_922.jpg'
 'BehavioralCloneData/IMG/left_2016_12_01_13_46_38_922.jpg'
 'BehavioralCloneData/IMG/right_2016_12_01_13_46_38_922.jpg']
[ 0.    0.25 -0.25 ...  0.    0.25 -0.25]

type(st[0])

numpy.float64

Now we have to get our Train and validation data

X_train,X_valid,y_train,y_valid=train_test_split(im,st,test_size=0.2,random_state=6)
print('Train size',len(X_train))
print('Validation size',len(X_valid))

Train size 9494
Validation size 2374

fig, axes = plt.subplots(1, 2, figsize=(12, 5))
axes[0].hist(y_train, bins=num_bins, width=0.05, color='blue')
axes[0].set_title('Training set')
axes[1].hist(y_valid, bins=num_bins, width=0.05, color='red')
axes[1].set_title('Validation set')

Text(0.5, 1.0, 'Validation set')

Image Preprocessing

Let's take any image, for example element 50

def preprocess(image):
  image=image[60:135,:,:]
  image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
  image = cv2.GaussianBlur(image,  (3, 3), 0)
  image = cv2.resize(image, (200, 66))
  image = image/255
  return image

any_image=im[50]
print(any_image)
original_image=mpimg.imread(any_image)
preprocessed_image=preprocess(original_image)
print(original_image.shape)
print(preprocessed_image.shape)

BehavioralCloneData/IMG/right_2016_12_01_13_32_45_377.jpg
(160, 320, 3)
(66, 200, 3)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()
axs[0].imshow(original_image)
axs[0].set_title('Original Image')
axs[1].imshow(preprocessed_image)
axs[1].set_title('Preprocessed Image')

Text(0.5, 1.0, 'Preprocessed Image')

Image Augmentation

I am going to perform a series of operations to augment the data

Zoom

def zoom(image):
  zoom=iaa.Affine(scale=(1,1.5))
  image=zoom.augment_image(image)
  return image

image = im[random.randint(0, 1000)]
original_image = mpimg.imread(image)
zoomed_image = zoom(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()
 
axs[0].imshow(original_image)
axs[0].set_title('Original Image')
axs[1].imshow(zoomed_image)
axs[1].set_title('Zoomed Image')

Text(0.5, 1.0, 'Zoomed Image')

def pan(image):
  pan = iaa.Affine(translate_percent= {"x" : (-0.1, 0.1), "y": (-0.1, 0.1)})
  image = pan.augment_image(image)
  return image

image = im[random.randint(0, 1000)]
original_image = mpimg.imread(image)
panned_image = pan(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()
 
axs[0].imshow(original_image)
axs[0].set_title('Original Image')
 
axs[1].imshow(panned_image)
axs[1].set_title('Panned Image')

Text(0.5, 1.0, 'Panned Image')

def img_random_brightness(image):
    brightness = iaa.Multiply((0.2, 1.2))
    image = brightness.augment_image(image)
    return image

image = im[random.randint(0, 1000)]
original_image = mpimg.imread(image)
brightness_altered_image = img_random_brightness(original_image)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()
 
axs[0].imshow(original_image)
axs[0].set_title('Original Image')
 
axs[1].imshow(brightness_altered_image)
axs[1].set_title('Brightness altered image ')

Text(0.5, 1.0, 'Brightness altered image ')

def img_random_flip(image, steering_angle):
    image = cv2.flip(image,1)
    steering_angle = -steering_angle
    return image, steering_angle

random_index = random.randint(0, 1000)
image = im[random_index]
steering_angle = st[random_index]
original_image = mpimg.imread(image)
flipped_image, flipped_steering_angle = img_random_flip(original_image, steering_angle)

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()
 
axs[0].imshow(original_image)
axs[0].set_title('Original Image - ' + 'Steering Angle:' + str(steering_angle))
 
axs[1].imshow(flipped_image)
axs[1].set_title('Flipped Image - ' + 'Steering Angle:' + str(flipped_steering_angle))

Text(0.5, 1.0, 'Flipped Image - Steering Angle:0.19780862999999999')

def random_augment(image, steering_angle):
    image = mpimg.imread(image)
    if np.random.rand() < 0.5:
      image = pan(image)
    if np.random.rand() < 0.5:
      image = zoom(image)
    if np.random.rand() < 0.5:
      image = img_random_brightness(image)
    if np.random.rand() < 0.5:
      image, steering_angle = img_random_flip(image, steering_angle)
    
    return image, steering_angle

ncol = 2
nrow = 10
 
fig, axs = plt.subplots(nrow, ncol, figsize=(15, 50))
fig.tight_layout()
 
for i in range(10):
  randnum = random.randint(0, len(im) - 1)
  random_image = im[randnum]
  random_steering = st[randnum]
    
  original_image = mpimg.imread(random_image)
  augmented_image, steering = random_augment(random_image, random_steering)
    
  axs[i][0].imshow(original_image)
  axs[i][0].set_title("Original Image")
  
  axs[i][1].imshow(augmented_image)
  axs[i][1].set_title("Augmented Image")

Batch Generator

def batch_generator(image_paths, steering_angles,batch_size,istraining):
  while True:
    batch_img=[]
    batch_steer=[]
    for i in range(batch_size):
      random_index = random.randint(0, len(image_paths) - 1)
      
      if istraining:
        image, steering = random_augment(image_paths[random_index], steering_angles[random_index])
      else:
        image=mpimg.imread(image_paths[random_index])
        steering=steering_angles[random_index]

      image=preprocess(image)
      batch_img.append(image)
      batch_steer.append(steering)

    yield np.asarray(batch_img),np.asarray(batch_steer)
  

Incorporate Data Augmentation

For example if I take a batch of 1 I have

x_train_gen, y_train_gen = next(batch_generator(X_train, y_train, 1,1))
x_valid_gen, y_valid_gen = next(batch_generator(X_valid, y_valid, 1,0))

fig, axs = plt.subplots(1, 2, figsize=(15, 10))
fig.tight_layout()
 
axs[0].imshow(x_train_gen[0])
axs[0].set_title('Training Image')
 
axs[1].imshow(x_valid_gen[0])
axs[1].set_title('Validation Image')

Text(0.5, 1.0, 'Validation Image')

The Model

I am going to implement NVIDIA model depicted in End-to-End Deep Learning for Self-Driving Cars

First let's check the input to verify it is in accordance to the shape of the Nvidia Model (66,200,3):

print("The shape of an input image is: ",x_train_gen[0].shape)

The shape of an input image is:  (66, 200, 3)

def nvidia_model():
  model=Sequential()

  #model.add(Lambda(preprocess, input_shape=(160,320,3)))

  model.add(Conv2D(24,kernel_size=(5,5),strides=(2,2),input_shape=(66,200,3),activation="elu"))
  model.add(Conv2D(36,kernel_size=(5,5),strides=(2,2),activation="elu"))
  model.add(Conv2D(48,kernel_size=(5,5),strides=(2,2),activation="elu"))

  model.add(Conv2D(64,kernel_size=(3,3),activation="elu"))
  model.add(Conv2D(64,kernel_size=(3,3),activation="elu"))
  #Dropout possible
  model.add(Dropout(0.5))

  model.add(Flatten())

  model.add(Dense(100, activation='elu'))
  #Dropout possible
  model.add(Dropout(0.5))

  model.add(Dense(50, activation='elu'))
  #Dropout possible
  model.add(Dropout(0.5))

  model.add(Dense(10, activation='elu'))
  #Dropout possible

  model.add(Dense(1))

  optimizer= Adam(lr=1e-3)
  model.compile(loss='mse', optimizer=optimizer)
  return model
 
 

model = nvidia_model()
print(model.summary())

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d (Conv2D)              (None, 31, 98, 24)        1824      
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 14, 47, 36)        21636     
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 5, 22, 48)         43248     
_________________________________________________________________
conv2d_3 (Conv2D)            (None, 3, 20, 64)         27712     
_________________________________________________________________
conv2d_4 (Conv2D)            (None, 1, 18, 64)         36928     
_________________________________________________________________
dropout (Dropout)            (None, 1, 18, 64)         0         
_________________________________________________________________
flatten (Flatten)            (None, 1152)              0         
_________________________________________________________________
dense (Dense)                (None, 100)               115300    
_________________________________________________________________
dropout_1 (Dropout)          (None, 100)               0         
_________________________________________________________________
dense_1 (Dense)              (None, 50)                5050      
_________________________________________________________________
dropout_2 (Dropout)          (None, 50)                0         
_________________________________________________________________
dense_2 (Dense)              (None, 10)                510       
_________________________________________________________________
dense_3 (Dense)              (None, 1)                 11        
=================================================================
Total params: 252,219
Trainable params: 252,219
Non-trainable params: 0
_________________________________________________________________
None

To train the model I use fit_generator

batch_size=100
epochs=30

history = model.fit(batch_generator(X_train, y_train, batch_size, 1),
                                  steps_per_epoch=300, 
                                  epochs=epochs,
                                  validation_data=batch_generator(X_valid, y_valid, batch_size, 0),
                                  validation_steps=200,
                                  verbose = 1,
                                  shuffle = 1)

Epoch 1/30
300/300 [==============================] - 183s 609ms/step - loss: 0.1011 - val_loss: 0.0469
Epoch 2/30
300/300 [==============================] - 183s 610ms/step - loss: 0.0584 - val_loss: 0.0406
Epoch 3/30
300/300 [==============================] - 183s 610ms/step - loss: 0.0559 - val_loss: 0.0391
Epoch 4/30
300/300 [==============================] - 184s 613ms/step - loss: 0.0534 - val_loss: 0.0373
Epoch 5/30
300/300 [==============================] - 183s 609ms/step - loss: 0.0510 - val_loss: 0.0357
Epoch 6/30
300/300 [==============================] - 184s 612ms/step - loss: 0.0497 - val_loss: 0.0335
Epoch 7/30
300/300 [==============================] - 185s 615ms/step - loss: 0.0470 - val_loss: 0.0325
Epoch 8/30
300/300 [==============================] - 184s 613ms/step - loss: 0.0458 - val_loss: 0.0322
Epoch 9/30
300/300 [==============================] - 184s 614ms/step - loss: 0.0441 - val_loss: 0.0303
Epoch 10/30
300/300 [==============================] - 185s 615ms/step - loss: 0.0439 - val_loss: 0.0322
Epoch 11/30
300/300 [==============================] - 184s 613ms/step - loss: 0.0435 - val_loss: 0.0324
Epoch 12/30
300/300 [==============================] - 184s 615ms/step - loss: 0.0422 - val_loss: 0.0319
Epoch 13/30
300/300 [==============================] - 185s 617ms/step - loss: 0.0421 - val_loss: 0.0336
Epoch 14/30
300/300 [==============================] - 185s 616ms/step - loss: 0.0406 - val_loss: 0.0298
Epoch 15/30
300/300 [==============================] - 185s 616ms/step - loss: 0.0394 - val_loss: 0.0309
Epoch 16/30
300/300 [==============================] - 185s 617ms/step - loss: 0.0384 - val_loss: 0.0311
Epoch 17/30
300/300 [==============================] - 185s 616ms/step - loss: 0.0384 - val_loss: 0.0305
Epoch 18/30
300/300 [==============================] - 184s 614ms/step - loss: 0.0378 - val_loss: 0.0333
Epoch 19/30
300/300 [==============================] - 184s 614ms/step - loss: 0.0366 - val_loss: 0.0295
Epoch 20/30
300/300 [==============================] - 184s 614ms/step - loss: 0.0373 - val_loss: 0.0265
Epoch 21/30
300/300 [==============================] - 184s 613ms/step - loss: 0.0359 - val_loss: 0.0258
Epoch 22/30
300/300 [==============================] - 185s 618ms/step - loss: 0.0357 - val_loss: 0.0253
Epoch 23/30
300/300 [==============================] - 187s 625ms/step - loss: 0.0351 - val_loss: 0.0285
Epoch 24/30
300/300 [==============================] - 187s 623ms/step - loss: 0.0346 - val_loss: 0.0272
Epoch 25/30
300/300 [==============================] - 187s 625ms/step - loss: 0.0343 - val_loss: 0.0257
Epoch 26/30
300/300 [==============================] - 188s 627ms/step - loss: 0.0339 - val_loss: 0.0241
Epoch 27/30
300/300 [==============================] - 189s 629ms/step - loss: 0.0337 - val_loss: 0.0270
Epoch 28/30
300/300 [==============================] - 189s 629ms/step - loss: 0.0331 - val_loss: 0.0253
Epoch 29/30
300/300 [==============================] - 191s 637ms/step - loss: 0.0331 - val_loss: 0.0251
Epoch 30/30
300/300 [==============================] - 191s 636ms/step - loss: 0.0331 - val_loss: 0.0258

# Model.fit_generator is deprecated and will be removed in future versions.
# Model.fit supports generators
#history = model.fit_generator(batch_generator(X_train, y_train, batch_size, 1),
#                                  steps_per_epoch=300, 
#                                  epochs=epochs,
#                                  validation_data=batch_generator(X_valid, y_valid, batch_size, 0),
#                                  validation_steps=200,
#                                  verbose = 1,
#                                  shuffle = 1)

#l=[0.0466,0.0191,0.0168,0.0123,0.0144,0.0134,0.0129,0.0121,0.0110,0.0102]
#v=[0.0226,0.0203,0.0199,0.0181,0.0159,0.0157,0.0151,0.0162,0.0149,0.0162]

#import matplotlib.pyplot as plt
#plt.plot(l)
#plt.plot(v)
#plt.title('model mean squared error loss')
#plt.ylabel('mean squared error loss')
#plt.xlabel('epoch')
#plt.ylim(0,0.25)
#plt.legend(['training set', 'validation set'], loc='upper right')
#plt.show()

plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.legend(['training', 'validation'])
plt.title('Loss')
plt.xlabel('Epoch')

Text(0.5, 0, 'Epoch')

model.save('modelWITHDrop3.h5')

from google.colab import files
files.download('modelWITHDrop3.h5')

(not used from here)

#!python --version

#import tensorflow as tf
#print(tf.__version__)

#keras.__version__

#cv2.__version__

#np.__version__